{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import re\n",
    "import numpy as np\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import time\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from unidecode import unidecode\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Negative</td>\n",
       "      <td>Lebih-lebih lagi dengan  kemudahan internet da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Positive</td>\n",
       "      <td>boleh memberi teguran kepada parti tetapi perl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Negative</td>\n",
       "      <td>Adalah membingungkan mengapa masyarakat Cina b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Positive</td>\n",
       "      <td>Kami menurunkan defisit daripada 6.7 peratus p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Negative</td>\n",
       "      <td>Ini masalahnya. Bukan rakyat, tetapi sistem</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                               text\n",
       "0  Negative  Lebih-lebih lagi dengan  kemudahan internet da...\n",
       "1  Positive  boleh memberi teguran kepada parti tetapi perl...\n",
       "2  Negative  Adalah membingungkan mengapa masyarakat Cina b...\n",
       "3  Positive  Kami menurunkan defisit daripada 6.7 peratus p...\n",
       "4  Negative        Ini masalahnya. Bukan rakyat, tetapi sistem"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('sentiment-news-bahasa-v5.csv')\n",
    "Y = LabelEncoder().fit_transform(df.label)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def textcleaning(string):\n",
    "    string = re.sub('http\\S+|www.\\S+', '',' '.join([i for i in string.split() if i.find('#')<0 and i.find('@')<0]))\n",
    "    string = unidecode(string).replace('.', '. ').replace(',', ', ')\n",
    "    string = re.sub('[^\\'\\\"A-Za-z\\- ]+', '', string)\n",
    "    return ' '.join([i for i in re.findall(\"[\\\\w']+|[;:\\-\\(\\)&.,!?\\\"]\", string) if len(i)>1]).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(df.shape[0]):\n",
    "    df.iloc[i,1] = textcleaning(df.iloc[i,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(df.text.values, Y, test_size = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic, maxlen, UNK=0):\n",
    "    X = np.zeros((len(corpus),maxlen))\n",
    "    for i in range(len(corpus)):\n",
    "        for no, k in enumerate(corpus[i].split()[:maxlen][::-1]):\n",
    "            try:\n",
    "                X[i,-1 - no]=dic[k]\n",
    "            except Exception as e:\n",
    "                X[i,-1 - no]=UNK\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('word2vec-256.p','rb') as fopen:\n",
    "    embedded = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def attention(inputs, attention_size):\n",
    "    hidden_size = inputs.shape[2].value\n",
    "    w_omega = tf.Variable(tf.random_normal([hidden_size, attention_size], stddev=0.1))\n",
    "    b_omega = tf.Variable(tf.random_normal([attention_size], stddev=0.1))\n",
    "    u_omega = tf.Variable(tf.random_normal([attention_size], stddev=0.1))\n",
    "    with tf.name_scope('v'):\n",
    "        v = tf.tanh(tf.tensordot(inputs, w_omega, axes=1) + b_omega)\n",
    "    vu = tf.tensordot(v, u_omega, axes=1, name='vu')\n",
    "    alphas = tf.nn.softmax(vu, name='alphas')\n",
    "    output = tf.reduce_sum(inputs * tf.expand_dims(alphas, -1), 1)\n",
    "    return output, alphas\n",
    "\n",
    "class Model:\n",
    "    def __init__(self, size_layer, num_layers, dropout, dimension_output, learning_rate, maxlen):\n",
    "        def cells(size, reuse=False):\n",
    "            return tf.contrib.rnn.DropoutWrapper(\n",
    "                tf.nn.rnn_cell.LSTMCell(size,initializer=tf.orthogonal_initializer(),reuse=reuse),\n",
    "                dropout,dropout,dropout)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        encoder_embeddings = tf.Variable(tf.convert_to_tensor(embedded['nce_weights'],\n",
    "                                                           dtype=tf.float32),trainable=False)\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        \n",
    "        for n in range(num_layers):\n",
    "            (out_fw, out_bw), (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = cells(size_layer),\n",
    "                cell_bw = cells(size_layer),\n",
    "                inputs = encoder_embedded,\n",
    "                dtype = tf.float32,\n",
    "                scope = 'bidirectional_rnn_%d'%(n))\n",
    "            encoder_embedded = tf.concat((out_fw, out_bw), 2)\n",
    "        self.outputs, self.attention = attention(encoder_embedded,maxlen)\n",
    "        W = tf.get_variable('w',shape=(size_layer*2, 2),initializer=tf.orthogonal_initializer())\n",
    "        b = tf.get_variable('b',shape=(2),initializer=tf.zeros_initializer())\n",
    "        self.logits = tf.add(tf.matmul(self.outputs, W),b,name='logits')\n",
    "        self.cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits = self.logits, \n",
    "                                                                           labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(tf.nn.in_top_k(self.logits, self.Y, 1), tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 512\n",
    "num_layers = 2\n",
    "dropout = 0.7\n",
    "dimension_output = 2\n",
    "learning_rate = 1e-4\n",
    "maxlen = 80\n",
    "batch_size = 16\n",
    "dictionary = embedded['dictionary']\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,num_layers,dropout,dimension_output,learning_rate,maxlen)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.701087\n",
      "time taken: 180.06061220169067\n",
      "epoch: 0, training loss: 0.625543, training acc: 0.653986, valid loss: 0.573572, valid acc: 0.701087\n",
      "\n",
      "time taken: 178.98020935058594\n",
      "epoch: 1, training loss: 0.553192, training acc: 0.716787, valid loss: 0.550418, valid acc: 0.701087\n",
      "\n",
      "time taken: 179.82218885421753\n",
      "epoch: 2, training loss: 0.522004, training acc: 0.736715, valid loss: 0.566059, valid acc: 0.690217\n",
      "\n",
      "time taken: 179.22341132164001\n",
      "epoch: 3, training loss: 0.506139, training acc: 0.737017, valid loss: 0.632580, valid acc: 0.695652\n",
      "\n",
      "epoch: 4, pass acc: 0.701087, current acc: 0.730978\n",
      "time taken: 179.15716218948364\n",
      "epoch: 4, training loss: 0.499618, training acc: 0.756944, valid loss: 0.592065, valid acc: 0.730978\n",
      "\n",
      "time taken: 180.52870917320251\n",
      "epoch: 5, training loss: 0.478605, training acc: 0.763889, valid loss: 0.561559, valid acc: 0.725543\n",
      "\n",
      "time taken: 179.04103302955627\n",
      "epoch: 6, training loss: 0.464527, training acc: 0.773853, valid loss: 0.571917, valid acc: 0.711957\n",
      "\n",
      "time taken: 180.0998501777649\n",
      "epoch: 7, training loss: 0.454069, training acc: 0.783514, valid loss: 0.595963, valid acc: 0.717391\n",
      "\n",
      "time taken: 178.92297959327698\n",
      "epoch: 8, training loss: 0.428208, training acc: 0.797403, valid loss: 0.646271, valid acc: 0.711957\n",
      "\n",
      "time taken: 179.03096866607666\n",
      "epoch: 9, training loss: 0.416883, training acc: 0.801329, valid loss: 0.644658, valid acc: 0.692935\n",
      "\n",
      "time taken: 179.86537098884583\n",
      "epoch: 10, training loss: 0.416720, training acc: 0.803442, valid loss: 0.628381, valid acc: 0.692935\n",
      "\n",
      "time taken: 179.11979961395264\n",
      "epoch: 11, training loss: 0.396514, training acc: 0.816425, valid loss: 0.620750, valid acc: 0.711957\n",
      "\n",
      "time taken: 179.83895635604858\n",
      "epoch: 12, training loss: 0.392566, training acc: 0.815217, valid loss: 0.628624, valid acc: 0.690217\n",
      "\n",
      "time taken: 146.43885731697083\n",
      "epoch: 13, training loss: 0.364218, training acc: 0.834239, valid loss: 0.652489, valid acc: 0.711957\n",
      "\n",
      "time taken: 82.6489565372467\n",
      "epoch: 14, training loss: 0.349485, training acc: 0.842089, valid loss: 0.622292, valid acc: 0.706522\n",
      "\n",
      "break epoch:15\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 10, 0, 0, 0\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n'%(EPOCH))\n",
    "        break\n",
    "    \n",
    "    train_X, train_Y = shuffle(train_X, train_Y)\n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    for i in range(0, (len(train_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(train_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : train_Y[i:i+batch_size]})\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "    \n",
    "    for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(test_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : test_Y[i:i+batch_size]})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "    \n",
    "    train_loss /= (len(train_X) // batch_size)\n",
    "    train_acc /= (len(train_X) // batch_size)\n",
    "    test_loss /= (len(test_X) // batch_size)\n",
    "    test_acc /= (len(test_X) // batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print('epoch: %d, pass acc: %f, current acc: %f'%(EPOCH,CURRENT_ACC, test_acc))\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "No variables to save",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-675f429867cc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0msaver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSaver\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetcwd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\"/attention/model.ckpt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, var_list, reshape, sharded, max_to_keep, keep_checkpoint_every_n_hours, name, restore_sequentially, saver_def, builder, defer_build, allow_empty, write_version, pad_step_number, save_relative_paths, filename)\u001b[0m\n\u001b[1;32m   1237\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdefer_build\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0min_graph_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1239\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1240\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaver_def\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1241\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_saver_def\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36mbuild\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1246\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0min_eager_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1247\u001b[0m       \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Use save/restore instead of build in eager mode.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1248\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuild_save\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuild_restore\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1249\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1250\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_build_eager\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheckpoint_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuild_save\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuild_restore\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36m_build\u001b[0;34m(self, checkpoint_path, build_save, build_restore)\u001b[0m\n\u001b[1;32m   1270\u001b[0m           \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1271\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1272\u001b[0;31m           \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"No variables to save\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1273\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_empty\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1274\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: No variables to save"
     ]
    }
   ],
   "source": [
    "import os\n",
    "saver = tf.train.Saver(tf.global_variables())\n",
    "saver.save(sess, os.getcwd()+\"/attention/model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings=','.join([n.name for n in tf.get_default_graph().as_graph_def().node if \"Variable\" in n.op or n.name.find('Placeholder') >= 0 or n.name.find('logits') == 0 or n.name.find('alphas') == 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            \"directory: %s\" % model_dir)\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "    \n",
    "    absolute_model_dir = \"/\".join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + \"/frozen_model.pb\"\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph=tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices)\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(\",\")\n",
    "        ) \n",
    "        with tf.gfile.GFile(output_graph, \"wb\") as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print(\"%d ops in the final graph.\" % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /home/barbatos/Desktop/rnn/attention/model.ckpt\n",
      "INFO:tensorflow:Froze 42 variables.\n",
      "Converted 42 variables to const ops.\n",
      "839 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph(\"attention\", strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, \"rb\") as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "g=load_graph('attention/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import/Placeholder\n",
      "import/Placeholder_1\n",
      "import/Variable\n",
      "import/Variable/read\n",
      "import/embedding_lookup\n",
      "import/bidirectional_rnn_0/fw/fw/Rank\n",
      "import/bidirectional_rnn_0/fw/fw/range/start\n",
      "import/bidirectional_rnn_0/fw/fw/range/delta\n",
      "import/bidirectional_rnn_0/fw/fw/range\n",
      "import/bidirectional_rnn_0/fw/fw/concat/values_0\n",
      "import/bidirectional_rnn_0/fw/fw/concat/axis\n",
      "import/bidirectional_rnn_0/fw/fw/concat\n",
      "import/bidirectional_rnn_0/fw/fw/transpose\n",
      "import/bidirectional_rnn_0/fw/fw/Shape\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice/stack\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice/stack_1\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice/stack_2\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims/dim\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/Const\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/concat/axis\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/concat\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/zeros/Const\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/zeros\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims_2/dim\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims_2\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/Const_2\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/concat_1/axis\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/concat_1\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/zeros_1/Const\n",
      "import/bidirectional_rnn_0/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/zeros_1\n",
      "import/bidirectional_rnn_0/fw/fw/Shape_1\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice_1/stack\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice_1/stack_1\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice_1/stack_2\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice_1\n",
      "import/bidirectional_rnn_0/fw/fw/time\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArray\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArray_1\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/Shape\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/strided_slice/stack\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/strided_slice/stack_1\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/strided_slice/stack_2\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/strided_slice\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/range/start\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/range/delta\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/range\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3\n",
      "import/bidirectional_rnn_0/fw/fw/while/iteration_counter\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/Less\n",
      "import/bidirectional_rnn_0/fw/fw/while/Less/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/Less_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/LogicalAnd\n",
      "import/bidirectional_rnn_0/fw/fw/while/LoopCond\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/add/y\n",
      "import/bidirectional_rnn_0/fw/fw/while/add\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayReadV3\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayReadV3/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayReadV3/Enter_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/keep_prob\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/Shape\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/random_uniform/min\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/random_uniform/max\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/random_uniform/sub\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/random_uniform/mul\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/random_uniform\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/add\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/Floor\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/div\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout/mul\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/kernel\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/kernel/read\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/bias\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/bias/read\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/concat/axis\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/concat\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/MatMul\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/MatMul/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/BiasAdd\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/BiasAdd/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/split/split_dim\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/split\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/add/y\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/add\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Sigmoid\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/mul\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Sigmoid_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Tanh\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/mul_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/add_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Sigmoid_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Tanh_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/mul_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/keep_prob\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/Shape\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/random_uniform/min\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/random_uniform/max\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/random_uniform/sub\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/random_uniform/mul\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/random_uniform\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/add\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/Floor\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/div\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_1/mul\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/keep_prob\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/Shape\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/random_uniform/min\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/random_uniform/max\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/random_uniform/sub\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/random_uniform/mul\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/random_uniform\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/add\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/Floor\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/div\n",
      "import/bidirectional_rnn_0/fw/fw/while/dropout_2/mul\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayWrite/TensorArrayWriteV3\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayWrite/TensorArrayWriteV3/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/add_1/y\n",
      "import/bidirectional_rnn_0/fw/fw/while/add_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/Exit_2\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/TensorArraySizeV3\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/range/start\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/range/delta\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/range\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/TensorArrayGatherV3\n",
      "import/bidirectional_rnn_0/fw/fw/Rank_1\n",
      "import/bidirectional_rnn_0/fw/fw/range_1/start\n",
      "import/bidirectional_rnn_0/fw/fw/range_1/delta\n",
      "import/bidirectional_rnn_0/fw/fw/range_1\n",
      "import/bidirectional_rnn_0/fw/fw/concat_2/values_0\n",
      "import/bidirectional_rnn_0/fw/fw/concat_2/axis\n",
      "import/bidirectional_rnn_0/fw/fw/concat_2\n",
      "import/bidirectional_rnn_0/fw/fw/transpose_1\n",
      "import/bidirectional_rnn_0/bw/ReverseV2/axis\n",
      "import/bidirectional_rnn_0/bw/ReverseV2\n",
      "import/bidirectional_rnn_0/bw/bw/Rank\n",
      "import/bidirectional_rnn_0/bw/bw/range/start\n",
      "import/bidirectional_rnn_0/bw/bw/range/delta\n",
      "import/bidirectional_rnn_0/bw/bw/range\n",
      "import/bidirectional_rnn_0/bw/bw/concat/values_0\n",
      "import/bidirectional_rnn_0/bw/bw/concat/axis\n",
      "import/bidirectional_rnn_0/bw/bw/concat\n",
      "import/bidirectional_rnn_0/bw/bw/transpose\n",
      "import/bidirectional_rnn_0/bw/bw/Shape\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice/stack\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice/stack_1\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice/stack_2\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims/dim\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/Const\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/concat/axis\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/concat\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/zeros/Const\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/zeros\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims_2/dim\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims_2\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/Const_2\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/concat_1/axis\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/concat_1\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/zeros_1/Const\n",
      "import/bidirectional_rnn_0/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/zeros_1\n",
      "import/bidirectional_rnn_0/bw/bw/Shape_1\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice_1/stack\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice_1/stack_1\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice_1/stack_2\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice_1\n",
      "import/bidirectional_rnn_0/bw/bw/time\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArray\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArray_1\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/Shape\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/strided_slice/stack\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/strided_slice/stack_1\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/strided_slice/stack_2\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/strided_slice\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/range/start\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/range/delta\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/range\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3\n",
      "import/bidirectional_rnn_0/bw/bw/while/iteration_counter\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/Less\n",
      "import/bidirectional_rnn_0/bw/bw/while/Less/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/Less_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/LogicalAnd\n",
      "import/bidirectional_rnn_0/bw/bw/while/LoopCond\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/add/y\n",
      "import/bidirectional_rnn_0/bw/bw/while/add\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayReadV3\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayReadV3/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayReadV3/Enter_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/keep_prob\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/Shape\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/random_uniform/min\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/random_uniform/max\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/random_uniform/sub\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/random_uniform/mul\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/random_uniform\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/add\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/Floor\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/div\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout/mul\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/kernel\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/kernel/read\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/bias\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/bias/read\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/concat/axis\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/concat\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/MatMul\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/MatMul/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/BiasAdd\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/BiasAdd/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/split/split_dim\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/split\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/add/y\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/add\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Sigmoid\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/mul\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Sigmoid_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Tanh\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/mul_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/add_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Sigmoid_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Tanh_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/mul_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/keep_prob\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/Shape\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/random_uniform/min\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/random_uniform/max\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/random_uniform/sub\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/random_uniform/mul\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/random_uniform\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/add\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/Floor\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/div\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_1/mul\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/keep_prob\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/Shape\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/random_uniform/min\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/random_uniform/max\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/random_uniform/sub\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/random_uniform/mul\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/random_uniform\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/add\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/Floor\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/div\n",
      "import/bidirectional_rnn_0/bw/bw/while/dropout_2/mul\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayWrite/TensorArrayWriteV3\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayWrite/TensorArrayWriteV3/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/add_1/y\n",
      "import/bidirectional_rnn_0/bw/bw/while/add_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/Exit_2\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/TensorArraySizeV3\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/range/start\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/range/delta\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/range\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/TensorArrayGatherV3\n",
      "import/bidirectional_rnn_0/bw/bw/Rank_1\n",
      "import/bidirectional_rnn_0/bw/bw/range_1/start\n",
      "import/bidirectional_rnn_0/bw/bw/range_1/delta\n",
      "import/bidirectional_rnn_0/bw/bw/range_1\n",
      "import/bidirectional_rnn_0/bw/bw/concat_2/values_0\n",
      "import/bidirectional_rnn_0/bw/bw/concat_2/axis\n",
      "import/bidirectional_rnn_0/bw/bw/concat_2\n",
      "import/bidirectional_rnn_0/bw/bw/transpose_1\n",
      "import/ReverseV2/axis\n",
      "import/ReverseV2\n",
      "import/concat/axis\n",
      "import/concat\n",
      "import/bidirectional_rnn_1/fw/fw/Rank\n",
      "import/bidirectional_rnn_1/fw/fw/range/start\n",
      "import/bidirectional_rnn_1/fw/fw/range/delta\n",
      "import/bidirectional_rnn_1/fw/fw/range\n",
      "import/bidirectional_rnn_1/fw/fw/concat/values_0\n",
      "import/bidirectional_rnn_1/fw/fw/concat/axis\n",
      "import/bidirectional_rnn_1/fw/fw/concat\n",
      "import/bidirectional_rnn_1/fw/fw/transpose\n",
      "import/bidirectional_rnn_1/fw/fw/Shape\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice/stack\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice/stack_1\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice/stack_2\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims/dim\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/Const\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/concat/axis\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/concat\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/zeros/Const\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/zeros\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims_2/dim\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims_2\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/Const_2\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/concat_1/axis\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/concat_1\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/zeros_1/Const\n",
      "import/bidirectional_rnn_1/fw/fw/DropoutWrapperZeroState/LSTMCellZeroState/zeros_1\n",
      "import/bidirectional_rnn_1/fw/fw/Shape_1\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice_1/stack\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice_1/stack_1\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice_1/stack_2\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice_1\n",
      "import/bidirectional_rnn_1/fw/fw/time\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArray\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArray_1\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/Shape\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/strided_slice/stack\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/strided_slice/stack_1\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/strided_slice/stack_2\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/strided_slice\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/range/start\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/range/delta\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/range\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3\n",
      "import/bidirectional_rnn_1/fw/fw/while/iteration_counter\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/Less\n",
      "import/bidirectional_rnn_1/fw/fw/while/Less/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/Less_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/LogicalAnd\n",
      "import/bidirectional_rnn_1/fw/fw/while/LoopCond\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/add/y\n",
      "import/bidirectional_rnn_1/fw/fw/while/add\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayReadV3\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayReadV3/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayReadV3/Enter_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/keep_prob\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/Shape\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/random_uniform/min\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/random_uniform/max\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/random_uniform/sub\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/random_uniform/mul\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/random_uniform\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/add\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/Floor\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/div\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout/mul\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/kernel\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/kernel/read\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/bias\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/bias/read\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/concat/axis\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/concat\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/MatMul\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/MatMul/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/BiasAdd\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/BiasAdd/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/split/split_dim\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/split\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/add/y\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/add\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Sigmoid\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/mul\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Sigmoid_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Tanh\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/mul_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/add_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Sigmoid_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Tanh_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/mul_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/keep_prob\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/Shape\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/random_uniform/min\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/random_uniform/max\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/random_uniform/sub\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/random_uniform/mul\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/random_uniform\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/add\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/Floor\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/div\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_1/mul\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/keep_prob\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/Shape\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/random_uniform/min\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/random_uniform/max\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/random_uniform/sub\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/random_uniform/mul\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/random_uniform\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/add\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/Floor\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/div\n",
      "import/bidirectional_rnn_1/fw/fw/while/dropout_2/mul\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayWrite/TensorArrayWriteV3\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayWrite/TensorArrayWriteV3/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/add_1/y\n",
      "import/bidirectional_rnn_1/fw/fw/while/add_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/Exit_2\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/TensorArraySizeV3\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/range/start\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/range/delta\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/range\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/TensorArrayGatherV3\n",
      "import/bidirectional_rnn_1/fw/fw/Rank_1\n",
      "import/bidirectional_rnn_1/fw/fw/range_1/start\n",
      "import/bidirectional_rnn_1/fw/fw/range_1/delta\n",
      "import/bidirectional_rnn_1/fw/fw/range_1\n",
      "import/bidirectional_rnn_1/fw/fw/concat_2/values_0\n",
      "import/bidirectional_rnn_1/fw/fw/concat_2/axis\n",
      "import/bidirectional_rnn_1/fw/fw/concat_2\n",
      "import/bidirectional_rnn_1/fw/fw/transpose_1\n",
      "import/bidirectional_rnn_1/bw/ReverseV2/axis\n",
      "import/bidirectional_rnn_1/bw/ReverseV2\n",
      "import/bidirectional_rnn_1/bw/bw/Rank\n",
      "import/bidirectional_rnn_1/bw/bw/range/start\n",
      "import/bidirectional_rnn_1/bw/bw/range/delta\n",
      "import/bidirectional_rnn_1/bw/bw/range\n",
      "import/bidirectional_rnn_1/bw/bw/concat/values_0\n",
      "import/bidirectional_rnn_1/bw/bw/concat/axis\n",
      "import/bidirectional_rnn_1/bw/bw/concat\n",
      "import/bidirectional_rnn_1/bw/bw/transpose\n",
      "import/bidirectional_rnn_1/bw/bw/Shape\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice/stack\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice/stack_1\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice/stack_2\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims/dim\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/Const\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/concat/axis\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/concat\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/zeros/Const\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/zeros\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims_2/dim\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/ExpandDims_2\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/Const_2\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/concat_1/axis\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/concat_1\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/zeros_1/Const\n",
      "import/bidirectional_rnn_1/bw/bw/DropoutWrapperZeroState/LSTMCellZeroState/zeros_1\n",
      "import/bidirectional_rnn_1/bw/bw/Shape_1\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice_1/stack\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice_1/stack_1\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice_1/stack_2\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice_1\n",
      "import/bidirectional_rnn_1/bw/bw/time\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArray\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArray_1\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/Shape\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/strided_slice/stack\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/strided_slice/stack_1\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/strided_slice/stack_2\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/strided_slice\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/range/start\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/range/delta\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/range\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3\n",
      "import/bidirectional_rnn_1/bw/bw/while/iteration_counter\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/Less\n",
      "import/bidirectional_rnn_1/bw/bw/while/Less/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/Less_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/LogicalAnd\n",
      "import/bidirectional_rnn_1/bw/bw/while/LoopCond\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/add/y\n",
      "import/bidirectional_rnn_1/bw/bw/while/add\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayReadV3\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayReadV3/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayReadV3/Enter_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/keep_prob\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/Shape\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/random_uniform/min\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/random_uniform/max\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/random_uniform/sub\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/random_uniform/mul\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/random_uniform\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/add\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/Floor\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/div\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout/mul\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/kernel\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/kernel/read\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/bias\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/bias/read\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/concat/axis\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/concat\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/MatMul\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/MatMul/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/BiasAdd\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/BiasAdd/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/split/split_dim\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/split\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/add/y\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/add\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Sigmoid\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/mul\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Sigmoid_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Tanh\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/mul_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/add_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Sigmoid_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Tanh_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/mul_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/keep_prob\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/Shape\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/random_uniform/min\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/random_uniform/max\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/random_uniform/sub\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/random_uniform/mul\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/random_uniform\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/add\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/Floor\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/div\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_1/mul\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/keep_prob\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/Shape\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/random_uniform/min\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/random_uniform/max\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/random_uniform/RandomUniform\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/random_uniform/sub\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/random_uniform/mul\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/random_uniform\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/add\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/Floor\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/div\n",
      "import/bidirectional_rnn_1/bw/bw/while/dropout_2/mul\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayWrite/TensorArrayWriteV3\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayWrite/TensorArrayWriteV3/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/add_1/y\n",
      "import/bidirectional_rnn_1/bw/bw/while/add_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/Exit_2\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/TensorArraySizeV3\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/range/start\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/range/delta\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/range\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/TensorArrayGatherV3\n",
      "import/bidirectional_rnn_1/bw/bw/Rank_1\n",
      "import/bidirectional_rnn_1/bw/bw/range_1/start\n",
      "import/bidirectional_rnn_1/bw/bw/range_1/delta\n",
      "import/bidirectional_rnn_1/bw/bw/range_1\n",
      "import/bidirectional_rnn_1/bw/bw/concat_2/values_0\n",
      "import/bidirectional_rnn_1/bw/bw/concat_2/axis\n",
      "import/bidirectional_rnn_1/bw/bw/concat_2\n",
      "import/bidirectional_rnn_1/bw/bw/transpose_1\n",
      "import/ReverseV2_1/axis\n",
      "import/ReverseV2_1\n",
      "import/concat_1/axis\n",
      "import/concat_1\n",
      "import/Variable_1\n",
      "import/Variable_1/read\n",
      "import/Variable_2\n",
      "import/Variable_2/read\n",
      "import/Variable_3\n",
      "import/Variable_3/read\n",
      "import/v/Tensordot/range/start\n",
      "import/v/Tensordot/range/limit\n",
      "import/v/Tensordot/range/delta\n",
      "import/v/Tensordot/range\n",
      "import/v/Tensordot/range_1/start\n",
      "import/v/Tensordot/range_1/limit\n",
      "import/v/Tensordot/range_1/delta\n",
      "import/v/Tensordot/range_1\n",
      "import/v/Tensordot/Shape\n",
      "import/v/Tensordot/Rank\n",
      "import/v/Tensordot/GreaterEqual/y\n",
      "import/v/Tensordot/GreaterEqual\n",
      "import/v/Tensordot/Cast\n",
      "import/v/Tensordot/mul\n",
      "import/v/Tensordot/Less/y\n",
      "import/v/Tensordot/Less\n",
      "import/v/Tensordot/Cast_1\n",
      "import/v/Tensordot/add\n",
      "import/v/Tensordot/mul_1\n",
      "import/v/Tensordot/add_1\n",
      "import/v/Tensordot/range_2/start\n",
      "import/v/Tensordot/range_2/delta\n",
      "import/v/Tensordot/range_2\n",
      "import/v/Tensordot/ListDiff\n",
      "import/v/Tensordot/Gather\n",
      "import/v/Tensordot/Gather_1\n",
      "import/v/Tensordot/Const\n",
      "import/v/Tensordot/Prod\n",
      "import/v/Tensordot/Const_1\n",
      "import/v/Tensordot/Prod_1\n",
      "import/v/Tensordot/concat_1/axis\n",
      "import/v/Tensordot/concat_1\n",
      "import/v/Tensordot/stack\n",
      "import/v/Tensordot/transpose\n",
      "import/v/Tensordot/Reshape\n",
      "import/v/Tensordot/Shape_1\n",
      "import/v/Tensordot/Rank_1\n",
      "import/v/Tensordot/GreaterEqual_1/y\n",
      "import/v/Tensordot/GreaterEqual_1\n",
      "import/v/Tensordot/Cast_2\n",
      "import/v/Tensordot/mul_2\n",
      "import/v/Tensordot/Less_1/y\n",
      "import/v/Tensordot/Less_1\n",
      "import/v/Tensordot/Cast_3\n",
      "import/v/Tensordot/add_2\n",
      "import/v/Tensordot/mul_3\n",
      "import/v/Tensordot/add_3\n",
      "import/v/Tensordot/range_3/start\n",
      "import/v/Tensordot/range_3/delta\n",
      "import/v/Tensordot/range_3\n",
      "import/v/Tensordot/ListDiff_1\n",
      "import/v/Tensordot/Gather_2\n",
      "import/v/Tensordot/Gather_3\n",
      "import/v/Tensordot/Const_2\n",
      "import/v/Tensordot/Prod_2\n",
      "import/v/Tensordot/Const_3\n",
      "import/v/Tensordot/Prod_3\n",
      "import/v/Tensordot/concat_3/axis\n",
      "import/v/Tensordot/concat_3\n",
      "import/v/Tensordot/stack_1\n",
      "import/v/Tensordot/transpose_1\n",
      "import/v/Tensordot/Reshape_1\n",
      "import/v/Tensordot/MatMul\n",
      "import/v/Tensordot/concat_4/axis\n",
      "import/v/Tensordot/concat_4\n",
      "import/v/Tensordot\n",
      "import/v/add\n",
      "import/v/Tanh\n",
      "import/vu/Rank\n",
      "import/vu/sub/y\n",
      "import/vu/sub\n",
      "import/vu/range/delta\n",
      "import/vu/range\n",
      "import/vu/range_1/start\n",
      "import/vu/range_1/limit\n",
      "import/vu/range_1/delta\n",
      "import/vu/range_1\n",
      "import/vu/Shape\n",
      "import/vu/Rank_1\n",
      "import/vu/GreaterEqual/y\n",
      "import/vu/GreaterEqual\n",
      "import/vu/Cast\n",
      "import/vu/mul\n",
      "import/vu/Less/y\n",
      "import/vu/Less\n",
      "import/vu/Cast_1\n",
      "import/vu/add\n",
      "import/vu/mul_1\n",
      "import/vu/add_1\n",
      "import/vu/range_2/start\n",
      "import/vu/range_2/delta\n",
      "import/vu/range_2\n",
      "import/vu/ListDiff\n",
      "import/vu/Gather\n",
      "import/vu/Gather_1\n",
      "import/vu/Const\n",
      "import/vu/Prod\n",
      "import/vu/Const_1\n",
      "import/vu/Prod_1\n",
      "import/vu/concat_1/axis\n",
      "import/vu/concat_1\n",
      "import/vu/stack\n",
      "import/vu/transpose\n",
      "import/vu/Reshape\n",
      "import/vu/Shape_1\n",
      "import/vu/Rank_2\n",
      "import/vu/GreaterEqual_1/y\n",
      "import/vu/GreaterEqual_1\n",
      "import/vu/Cast_2\n",
      "import/vu/mul_2\n",
      "import/vu/Less_1/y\n",
      "import/vu/Less_1\n",
      "import/vu/Cast_3\n",
      "import/vu/add_2\n",
      "import/vu/mul_3\n",
      "import/vu/add_3\n",
      "import/vu/range_3/start\n",
      "import/vu/range_3/delta\n",
      "import/vu/range_3\n",
      "import/vu/ListDiff_1\n",
      "import/vu/Gather_2\n",
      "import/vu/Gather_3\n",
      "import/vu/Const_2\n",
      "import/vu/Prod_2\n",
      "import/vu/Const_3\n",
      "import/vu/Prod_3\n",
      "import/vu/concat_3/axis\n",
      "import/vu/concat_3\n",
      "import/vu/stack_1\n",
      "import/vu/transpose_1\n",
      "import/vu/Reshape_1\n",
      "import/vu/MatMul\n",
      "import/vu/concat_4/axis\n",
      "import/vu/concat_4\n",
      "import/vu\n",
      "import/Shape\n",
      "import/Rank\n",
      "import/Shape_1\n",
      "import/Sub/y\n",
      "import/Sub\n",
      "import/Slice/begin\n",
      "import/Slice/size\n",
      "import/Slice\n",
      "import/concat_2/values_0\n",
      "import/concat_2/axis\n",
      "import/concat_2\n",
      "import/Reshape\n",
      "import/Softmax\n",
      "import/alphas\n",
      "import/ExpandDims/dim\n",
      "import/ExpandDims\n",
      "import/mul\n",
      "import/Sum/reduction_indices\n",
      "import/Sum\n",
      "import/w\n",
      "import/w/read\n",
      "import/b\n",
      "import/b/read\n",
      "import/MatMul\n",
      "import/logits\n",
      "import/beta1_power\n",
      "import/beta2_power\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/kernel/Adam\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/kernel/Adam_1\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/bias/Adam\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/bias/Adam_1\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/kernel/Adam\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/kernel/Adam_1\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/bias/Adam\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/bias/Adam_1\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/kernel/Adam\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/kernel/Adam_1\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/bias/Adam\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/bias/Adam_1\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/kernel/Adam\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/kernel/Adam_1\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/bias/Adam\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/bias/Adam_1\n",
      "import/Variable_1/Adam\n",
      "import/Variable_1/Adam_1\n",
      "import/Variable_2/Adam\n",
      "import/Variable_2/Adam_1\n",
      "import/Variable_3/Adam\n",
      "import/Variable_3/Adam_1\n",
      "import/w/Adam\n",
      "import/w/Adam_1\n",
      "import/b/Adam\n",
      "import/b/Adam_1\n"
     ]
    }
   ],
   "source": [
    "for op in g.get_operations():\n",
    "    print(op.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16, 80)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "alphas = g.get_tensor_by_name('import/alphas:0')\n",
    "test_sess = tf.InteractiveSession(graph=g)\n",
    "test_sess.run([logits,alphas], feed_dict={x:batch_x})[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
